/**
 * 采集 haouc
 * @link http://hao.uc.cn
 */
const fs = require('fs');
const axios = require('axios')
const cheerio = require('cheerio')
// Node 环境当中不支持 GBK 编码，所以需要引用 iconv-lite 模块来转码
const iconv = require('iconv-lite')

axios.defaults.timeout = 10000; // 10秒超时
const SITE_NAME = 'haouc';
const RUNTIME_PATH = './runtime/' + SITE_NAME + '/';
const SQL_FILE_PATH = RUNTIME_PATH + SITE_NAME + '.sql';
const JSON_FILE_PATH = RUNTIME_PATH + SITE_NAME + '.json';
if (!fs.existsSync(RUNTIME_PATH)) {
    fs.mkdirSync(RUNTIME_PATH)
}

// getList();
// return;
(async function () {
    let links = [
        {
            "name": "xinwen",
            "text": "新闻",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33270&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "tiyu",
            "text": "体育",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33272&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "junshi",
            "text": "军事",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33271&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "qiche",
            "text": "汽车",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33273&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "keji",
            "text": "科技",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33274&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "caijing",
            "text": "财经",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33275&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "licai",
            "text": "理财",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33208&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "yule",
            "text": "娱乐",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33278&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "xiaoshuo",
            "text": "小说",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33209&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "gaoxiao",
            "text": "搞笑",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33277&fcid=2016&cid=1117&page=navi_online_index"
        },
        // {
        //     "name": "dongman",
        //     "text": "动漫",
        //     "link": "http://stat.hao.uc.cn/stat_site.php?nid=39989&fcid=2016&cid=1117&page=navi_online_index"
        // },
        {
            "name": "xingzuo",
            "text": "星座",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33210&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "yinyue",
            "text": "音乐",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34556&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "shequ",
            "text": "社区",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34554&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "nvxing",
            "text": "女性",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34190&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "shiping",
            "text": "视频",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=39396&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "gouwu",
            "text": "购物",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34558&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "jiaoyu",
            "text": "教育",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33207&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "caipiao",
            "text": "彩票",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34555&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "lvyou",
            "text": "旅游",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33211&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "meishi",
            "text": "美食",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34193&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "youxiang",
            "text": "邮箱",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34557&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "zhaopin",
            "text": "招聘",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34192&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "yinhang",
            "text": "银行",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34189&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "jiankang",
            "text": "健康",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=33276&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "fangchan",
            "text": "房产",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34194&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "shenghuo",
            "text": "生活",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34561&fcid=2016&cid=1117&page=navi_online_index"
        },
        {
            "name": "zonghe",
            "text": "综合",
            "link": "http://stat.hao.uc.cn/stat_site.php?nid=34559&fcid=2016&cid=1117&page=navi_online_index"
        }
    ];
    let list = [];
    for (let item of links) {
        list = list.concat(await getJson(item.name, item.text, item.link));
    }
    fs.writeFileSync(JSON_FILE_PATH, JSON.stringify(list));
    fs.writeFileSync(SQL_FILE_PATH, json2sql(list).join("\n"))
    console.log('列表采集完毕.');
})()


async function getJson(name, text, url) {
    console.log(text);
    let tempFile = RUNTIME_PATH + name + ".html", html = '';
    if (!fs.existsSync(tempFile)) {
        html = (await axios.get(url)).data;
        fs.writeFileSync(tempFile, html);
    } else {
        html = fs.readFileSync(tempFile);
    }
    // html=iconv.decode(html,'gbk');
    let $ = cheerio.load(html);
    let data = { text: text, data: [] };
    $('section').each(function (i, el) {
        if (i == 0) {
            return;
        }
        let $el = $(el), $elFirst = $el.find('h2');
        if (['时评图论'].includes($elFirst.text())) {
            return;
        }
        let item = { text: $elFirst.text().trim(), data: [] };
        $el.find('li a').each(function (i2, el2) {
            let $el2 = $(el2);
            item.children.push({ text: $el2.text().trim(), link: $el2.attr('href') });
        })
        data.children.push(item);
        // console.log(data);
    })
    return data;
}
function json2sql(json) {
    let sqlArr = [], table = 'tbl_navigate', id = 0, pid = 0;
    json.forEach(item => {
        id++;
        sqlArr.push(`INSERT INTO ${table} (id, pid, text, link) VALUES ('${id}', 0, '${item.text}', '${item.link}');`);
        pid = id;
        item.children.forEach(item2 => {
            id++;
            sqlArr.push(`INSERT INTO ${table} (id, pid, text, link) VALUES ('${id}', ${pid}, '${item2.text}', '${item2.link}');`);
            pid = id;
            item2.children.forEach(item3 => {
                id++;
                sqlArr.push(`INSERT INTO ${table} (id, pid, text, link) VALUES ('${id}', ${pid}, '${item3.text}', '${item3.link}');`);
            })
        })
    })
    return sqlArr;
}


function getList() {
    let html = fs.readFileSync(RUNTIME_PATH + 'uc.html');
    let $ = cheerio.load(html), list = [];
    $('a.linkList-item').each(function (i, el) {
        let $el = $(el);
        list.push({ name: '', text: $el.text(), link: $el.attr('href') });
    })
    fs.writeFileSync(RUNTIME_PATH + 'uc.json', JSON.stringify(list));
}